import json
import os
import platform
import re
import sys
import threading
import time
from urllib import parse

import click
import facade
import requests
from vip.DiscernCode import DiscernCode
from xjlibrary.mdatetime.mtime2 import MDateTimeUtils
from xjlibrary.mprocesspoll.process_distributed.BaseTask import BaseTask
from xjlibrary.network.MyIP import GetLocalIPByPrefix, get_local_ip
from xjlibrary.our_file_dir import BaseDir
from xjlibrary.tools.BaseUrl import BaseUrl

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
dirPath = BaseDir.get_new_path(TopPath, "download", "EI", "download", "json", "refs")
BaseDir.create_dir(dirPath)
dbpath = BaseDir.get_upper_dir(curPath, -1)

"""
多进程版
"""


class NodeTask(BaseTask):
    def __init__(self):
        super().__init__()
        # 连接到服务器，也就是运行task_master.py的机器:
        sysstr = platform.system()
        if sysstr == "Windows":
            self.ip = GetLocalIPByPrefix("192.168.")
        else:
            self.ip = get_local_ip("enp2s0")
        self.server_addr = self.ip
        self.port = 5000
        self.init_node(self.server_addr, self.port)


class DownRefs(object):
    # 可用ip
    listip = [
        "192.168.30.176:8207",
        "192.168.30.176:8012",
        "192.168.30.176:8021",
        "192.168.30.36:8041",  ##
        "192.168.30.176:8076",
        "192.168.30.36:8082",  ##
        "192.168.30.176:8165",
        "192.168.30.176:8171",
        "192.168.30.36:8182",  ##
        "192.168.30.176:8195",
        "192.168.30.176:8243",
        "192.168.30.176:8031",
    ]

    def __init__(self):
        self.configfile = BaseDir.get_new_path(dbpath, "db.ini")
        self.logger = facade.get_streamlogger()
        # self.mysqlutils = facade.MysqlUtiles(self.configfile, "db", logger=self.logger)
        self.Proxies = None
        self.discern_code_time = int(time.time()) - 300
        self.sn = requests.Session()
        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
        self.sn.headers['User-Agent'] = self.UserAgent
        self.t = int(round(time.time() * 1000))
        self.SEARCHID = ""
        self.init_form()
        self.node = NodeTask()
        self.init_node()

    def init_node(self):
        self.node.conn()
        self.node.set_task_result_obj()

    def set_proxy(self, proxy):
        self.Proxies = {
            "http": proxy,
            "https": proxy  # key是指目标网站的协议
        }

    def init_form(self):
        self.Form = {
            "usageOrigin": "searchform",
            "usageZone": "expertsearch",
            "editSearch": "",
            "isFullJsonResult": "true",
            "angularReq": "true",
            "CID": "searchSubmit",
            "searchtype": "Expert",
            "origin": "searchform",
            "category": "expertsearch",
            "searchWord1": "((1994*) WN AN)",
            "allDb": "1",
            "database": "1",
            "yearselect": "yearrange",
            "startYear": "1884",
            "endYear": "2020",
            "updatesNo": "1",  # "4",
            "sort": "yr",  # "sort": "relevance",  yr代表最新 relevance相关度排序
            "autostem": "true",
            "searchStartTimestamp": str(int(round(time.time() * 1000))),
            "_": str(self.t)
        }
        # ref 请求 为get请求   self.refsgetdata 为请求参数
        self.refsurl = "https://www.engineeringvillage.com/search/doc/refs.url"
        self.refsgetdata = {
            "content": "true",
            "refType": "compendex",
            "searchtype": "Expert",
            "usageOrigin": "recordpage",
            "usageZone": "abstracttab",
            "pageType": "expertSearch",
            "SEARCHID": "07c12a411884442687a6e19f410cdf13",
            "DOCINDEX": "1",
            "database": "1",
            "docid": "cpx_M2a23a19e16b1f38b06bM72b710178163167",
            "totalResultsCount": "1",
            "displayPagination": "yes",
            "dbid": "cpx"
        }
        # 获取剩下所有的引文
        self.refsshowall = {
            "content": "true",
            "compendexajax": "t",
            "docid": "cpx_M2a23a19e16b1f38b06bM72b710178163167",
            "SEARCHID": "07c12a411884442687a6e19f410cdf13",
            "database": "1",
            "DOCINDEX": "",
            "currPageNumber": "2",
            "searchtype": "Expert",
            "pageSize": "25"
        }

        self.headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Connection": "keep-alive",
            "Content-Type": "application/json",
            "Host": "www.engineeringvillage.com",
            "Referer": "https://www.engineeringvillage.com/search/expert.url",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
            "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
            "X-Requested-With": "XMLHttpRequest"
        }

    def discern_code(self, jpgpath):
        """
        识别验证码通过云打码
        :return:
        """
        if int(time.time()) - self.discern_code_time > 60 * 5:  #
            boolresult, code = DiscernCode(self.logger).discern_code(jpgpath)
            BaseDir.single_add_file("./code.txt", "***code is {}:{}\n".format(os.getpid(), code))
            self.discern_code_time = int(time.time())
            if boolresult:
                return True, code
            else:
                return False, ""
        else:
            self.logger.info("时间还没到休息60秒 差时为{} 应该相隔5分钟".format(int(time.time()) - self.discern_code_time))
            time.sleep(60)
            return False, ""

    def get_searchid(self):
        self.logger.info('get_searchid')
        self.sn = requests.Session()
        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
        self.sn.headers['User-Agent'] = self.UserAgent

        # 如果目录存在表示下载过 但也有可能是下载一部分后中断了
        # outDir = os.path.join(dirPath, ac)
        # if os.path.exists(outDir):
        #     return

        def init_home():
            url = r'https://www.engineeringvillage.com/home.url'
            header = {
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                "Accept-Encoding": "gzip, deflate",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                "Host": "www.engineeringvillage.com",
                "Upgrade-Insecure-Requests": "1",
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
            }
            BoolResult, errString, r = facade.BaseRequest(url,
                                                          sn=self.sn,
                                                          headers=header,
                                                          verify=False,
                                                          proxies=self.Proxies,
                                                          timeout=60)
            self.logger.info(errString)
            if not BoolResult:
                return False
            self.logger.info("首页访问完成")
            print(r.url)
            BaseDir.single_write_file(r.text, "./test.html")
            if r.text.find("Chongqing University Library, General Access") > -1:
                obj = re.search(r'csrfToken\.value = "(.*?)"', r.text)
                csrfToken = obj.group(1)
                print(csrfToken)
                url = "https://www.engineeringvillage.com/customer/authenticate.url"
                postdate = {
                    "path_choice": "13022631",
                    "remember_path_flag": "true",
                    "origin": "pathChoice",
                    "zone": "main",
                    "auth_type": "CANDIDATE_PATH",
                    "csrfToken": csrfToken
                }
                BoolResult, errString, r = facade.BaseRequestPost(url=url,
                                                                  sn=self.sn,
                                                                  data=postdate,
                                                                  proxies=self.Proxies,
                                                                  allow_redirects=True,
                                                                  endstring="",
                                                                  verify=False,
                                                                  timeout=60)
                if BoolResult:
                    print("suceess")
                else:
                    print("failed")
                    sys.exit(-1)

        form = self.Form
        # 5846789  17977686
        form['searchWord1'] = "china WN ALL"
        # form['startYear'] = str(year)
        #         # form['endYear'] = str(year)
        url = r'https://www.engineeringvillage.com/search/submit.url'
        url = url + "?" + BaseUrl.dicts_to_url(form)
        BoolResult, errString, r = facade.BaseRequest(url=url,
                                                      sn=self.sn,
                                                      headers=self.headers,
                                                      proxies=self.Proxies,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      verify=False,
                                                      timeout=60)
        self.logger.info(errString)

        # self.logger.info(r.status_code)
        if errString == "request":
            return False
        # if r and r.status_code == 200:
        #
        if not BoolResult:
            if r.status_code == 400:
                imagetextenc = json.loads(r.text)["imagetextenc"]
                url = "https://www.engineeringvillage.com/rest/captcha/image?imagetextenc={}".format(imagetextenc)
                BoolResult, errString, r = facade.BaseRequest(url=url,
                                                              sn=self.sn,
                                                              proxies=self.Proxies,
                                                              allow_redirects=True,
                                                              endstring="",
                                                              verify=False,
                                                              timeout=60)
                if BoolResult:
                    jpgpath = BaseDir.get_new_path(curPath, str(os.getpid()) + ".jpg")
                    BaseDir.single_write_wb_file(r.content, jpgpath)
                else:
                    print("获取验证码失败")
                    return False
                # code = input("please input code:")
                jpgpath = BaseDir.get_new_path(curPath, str(os.getpid()) + ".jpg")
                bools, code = self.discern_code(jpgpath)
                if not bools:
                    return False
                url = "https://www.engineeringvillage.com/rest/captcha/verify?imagetextenc={}&userentry={}".format(
                    imagetextenc, code)
                BoolResult, errString, r = facade.BaseRequest(url=url,
                                                              sn=self.sn,
                                                              proxies=self.Proxies,
                                                              allow_redirects=True,
                                                              endstring="",
                                                              verify=False,
                                                              timeout=60)
                if BoolResult:
                    url = r'https://www.engineeringvillage.com/search/submit.url'
                    url = url + "?" + BaseUrl.dicts_to_url(form)
                    BoolResult, errString, r = facade.BaseRequest(url=url,
                                                                  sn=self.sn,
                                                                  proxies=self.Proxies,
                                                                  allow_redirects=True,
                                                                  endstring="",
                                                                  verify=False,
                                                                  timeout=60)
                    if not BoolResult:
                        print("验证后失败")
                        return False
                else:
                    print("验证失败")
                    return False
            else:
                return False
        self.logger.info("搜索成功")
        print(r.url)
        BaseDir.single_write_file(r.text, "./test2.html")
        try:
            param_dict = parse.parse_qs(parse.urlparse(r.url).query)
            self.SEARCHID = param_dict['SEARCHID'][0]
            return True
        except:
            init_home()
            return False

    def down_one(self, docid, ac, refcount):
        downdate = MDateTimeUtils.get_today_date_strings()
        outPathFile = os.path.join(dirPath,
                                   '%s_%s_%d_%d.big_json' % (
                                       downdate, GetLocalIPByPrefix('192.168.'), os.getpid(),
                                       threading.get_ident())
                                   )
        self.refsgetdata["SEARCHID"] = self.SEARCHID
        self.refsgetdata["docid"] = docid
        # 开始下载引文
        urlpara = BaseUrl.dicts_to_url(self.refsgetdata)
        refsurl_full = self.refsurl + "?" + urlpara
        allcount = 0
        refdicts = {}
        BoolResult, errString, r = facade.BaseRequest(refsurl_full,
                                                      sn=self.sn,
                                                      mark="patrefcount",
                                                      proxies=self.Proxies,
                                                      headers=self.headers,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      verify=False,
                                                      timeout=60)

        if BoolResult:
            try:
                dicts = json.loads(r.text)
            except:
                return "break"
            allcount = int(dicts["result"]["abstractrecord"]["patrefcount"])
            refdicts["downdate"] = downdate
            refdicts["rawid"] = ac
            refdicts["onepage"] = r.text
        else:
            sql = "update articlenew set ref_stat=-1  where AccessionNumber='{}'".format(ac)
            # self.mysqlutils.ExeSqlToDB(sql)
            self.node.result.put(sql)
            if r and r.status_code == 500:
                return "break"

        if str(refcount) != str(allcount):
            sql = "update articlenew set ref_cnt='{}' where AccessionNumber='{}'".format(str(allcount), ac)
            # self.mysqlutils.ExeSqlToDB(sql)
            self.node.result.put(sql)

        if allcount > 25:
            self.refsshowall["SEARCHID"] = self.SEARCHID
            self.refsshowall["docid"] = docid
            urlpara = BaseUrl.dicts_to_url(self.refsshowall)
            refsurlall_full = self.refsurl + "?" + urlpara
            BoolResult, errString, r = facade.BaseRequest(refsurlall_full,
                                                          sn=self.sn,
                                                          mark="patrefcount",
                                                          proxies=self.Proxies,
                                                          headers=self.headers,
                                                          allow_redirects=True,
                                                          endstring="",
                                                          verify=False,
                                                          timeout=(30, 60))
            if BoolResult:
                try:
                    dicts = json.loads(r.text)
                except:
                    return "break"
                allcount = int(dicts["result"]["abstractrecord"]["patrefcount"])
                refdicts["twopage"] = r.text
                jsonstring = json.dumps(refdicts, ensure_ascii=False)
                BaseDir.single_add_file(outPathFile, jsonstring + "\n")
                sql = "update articlenew set ref_stat=1  where AccessionNumber='{}'".format(ac)
                # self.mysqlutils.ExeSqlToDB(sql)
                self.node.result.put(sql)
            else:
                sql = "update articlenew set ref_stat=-1  where AccessionNumber='{}'".format(ac)
                # self.mysqlutils.ExeSqlToDB(sql)
                self.node.result.put(sql)

        else:
            if str(allcount) != "-1":
                refdicts["twopage"] = ""
                jsonstring = json.dumps(refdicts, ensure_ascii=False)
                BaseDir.single_add_file(outPathFile, jsonstring + "\n")

            sql = "update articlenew set ref_stat=1  where AccessionNumber='{}'".format(ac)
            # self.mysqlutils.ExeSqlToDB(sql)
            self.node.result.put(sql)

    # def select(self):
    #     sql = "SELECT AccessionNumber,docid,ref_cnt FROM `ei`.`articlenew` WHERE `country` = 'CN' AND `ref_stat` = 0 ORDER BY `ref_cnt` DESC limit 1000"
    #     # sql = "SELECT AccessionNumber,docid,ref_cnt FROM `ei`.`articlenew` WHERE AccessionNumber='20133816759084'"
    #     rows = self.mysqlutils.SelectFromDB(sql)
    #     return rows

    def control(self):

        while True:
            if self.get_searchid():
                for _ in range(1000):
                    ids = self.node.task.get()
                    AccessionNumber, docid, ref_cnt = ids
                    results = self.down_one(docid, AccessionNumber, ref_cnt)
                    if results == "break":
                        break


@click.command()
@click.option('--proxy', default='192.168.30.176:8041', type=str, help='请输入目录路径的列表')
def main(proxy):
    print("inputs is %s " % proxy)
    down = DownRefs()
    down.set_proxy(proxy)
    down.control()


if __name__ == "__main__":
    main()

    # down.set_proxy("192.168.30.176:8012")
    # down.down_one("cpx_6e3d601415616ba2bM426a2061377553", "20133816759084")
    # down.down_one("cpx_6e3d601415616ba2bM426a2061377553", "20133816759084")
